{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.00720062,  1.4170165 , -0.7293671 ,  0.27093425,  0.00835057,\n",
       "        0.1652125 ,  0.        ,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('LunarLander-v2',\n",
    "                       continuous=True,\n",
    "                       render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZWklEQVR4nO3de0yUd/4v8Pfcuc6M3GagDNWqiIi6ihemxu5mpeJlXfGS0MatrDE1Inbr2ja7nOxu0z1J8bSptbb2ErO15vdL68ae2J5Y264Hb2ulKghVEamxq1BloIrMoHKfz/mD8pyORQVFvoy+X8kndZ7vd2Y+z9OZN888D8+gExEBEZECetUNENGDiwFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKKAugTZs2YejQoQgJCcHUqVNx5MgRVa0QkSJKAuif//wn1q5dixdffBHHjh3D+PHjkZWVhfr6ehXtEJEiOhUXo06dOhWTJ0/GW2+9BQDw+/1wuVx45pln8Oc//3mg2yEiRYwD/YRtbW0oLS1FQUGBtkyv1yMzMxPFxcU93qe1tRWtra3abb/fj4aGBkRHR0On093znomob0QETU1NSEhIgF5/8w9aAx5Aly5dQmdnJxwOR8Byh8OB06dP93ifwsJCvPTSSwPRHhH1o5qaGiQmJt50fMAD6E4UFBRg7dq12m2v14ukpCSFHdHtDE3MwG+m/y9cb72E/7P3TwizDMHsX/5PDAkd1vsHET++OvUm/n30bYwcOgOPT/0faGquw//+v6tx/XrDvWue+k1kZOQtxwf8IHRMTAwMBgPq6uoCltfV1cHpdPZ4H4vFAqvVqpXNZhuIVukOGY0hmJj6BCJDHTj1n524fOUs9HojjEYT9AY99AYd9AYddHr8WAKdXgC9AHr/j9UJ0fuhN3b9jDz3/SFU1x+GK3Yyxo6aD4AfvYPB7Q6RDPgekNlsRnp6OoqKipCdnQ2g65hOUVERVq9ePdDt0D3gip+AYfGP4UJDGY5XfQqB4IqvGgeOvQEdDNBB95P80P34T93Pb+uAusunAADtHc04VvERhsa7MSH5CZw5V4SGK9UDu2LU75R8BFu7di1yc3MxadIkTJkyBRs2bMC1a9ewbNkyFe1QPxtiS0JTy0UcOblF+6jkvXoBJyov3NXj1v5wCiWn/wsGgxnNLb7+aJVUE0XefPNNSUpKErPZLFOmTJGvv/661/f1er0CgDVIy6A3i8uZLkZjSP8/tsEker1B+Tqyelder/eW72Ulvwd0t3w+H48DEQUBr9cLq9V603FeC0ZEyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJl+hxABw4cwLx585CQkACdTodPPvkkYFxE8Le//Q3x8fEIDQ1FZmYmzpw5EzCnoaEBS5YsgdVqhd1ux/Lly3H16tW7WhEiCj59DqBr165h/Pjx2LRpU4/jr7zyCjZu3Ih3330Xhw8fRnh4OLKystDS0qLNWbJkCSoqKrB7927s3LkTBw4cwIoVK+58LYgoOMldACA7duzQbvv9fnE6nfLqq69qyxobG8VischHH30kIiKnTp0SAHL06FFtzueffy46nU4uXLjQq+f1er0CgMViDfLyer23fC/36zGg//znP/B4PMjMzNSW2Ww2TJ06FcXFxQCA4uJi2O12TJo0SZuTmZkJvV6Pw4cP9/i4ra2t8Pl8AUVEwa9fA8jj8QAAHA5HwHKHw6GNeTwexMXFBYwbjUZERUVpc25UWFgIm82mlcvl6s+2iUiRoDgLVlBQAK/Xq1VNTY3qloioH/RrADmdTgBAXV1dwPK6ujptzOl0or6+PmC8o6MDDQ0N2pwbWSwWWK3WgCKi4NevATRs2DA4nU4UFRVpy3w+Hw4fPgy32w0AcLvdaGxsRGlpqTZnz5498Pv9mDp1an+2Q0SDXR9OeomISFNTk5SVlUlZWZkAkPXr10tZWZmcP39eRETWrVsndrtdPv30Uzl+/LjMnz9fhg0bJs3NzdpjzJo1SyZMmCCHDx+WgwcPysiRI+XJJ5/sdQ88C8ZiBUfd7ixYnwNo7969PT5Rbm6uiHSdiv/rX/8qDodDLBaLzJgxQ6qqqgIe4/Lly/Lkk09KRESEWK1WWbZsmTQ1NTGAWKz7rG4XQDoREQQZn88Hm82mug0iug2v13vLY7ZBcRaMiO5PDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJk+BVBhYSEmT56MyMhIxMXFITs7G1VVVQFzWlpakJ+fj+joaERERGDRokWoq6sLmFNdXY25c+ciLCwMcXFxeOGFF9DR0XH3a0NEwUX6ICsrS7Zs2SInT56U8vJymTNnjiQlJcnVq1e1OStXrhSXyyVFRUVSUlIiGRkZ8uijj2rjHR0dkpaWJpmZmVJWVia7du2SmJgYKSgo6HUfXq9XALBYrEFeXq/3lu/lPgXQjerr6wWA7N+/X0REGhsbxWQyyfbt27U5lZWVAkCKi4tFRGTXrl2i1+vF4/Foc9555x2xWq3S2traq+dlALFYwVG3C6C7Ogbk9XoBAFFRUQCA0tJStLe3IzMzU5uTkpKCpKQkFBcXAwCKi4sxduxYOBwObU5WVhZ8Ph8qKip6fJ7W1lb4fL6AIqLgd8cB5Pf7sWbNGkybNg1paWkAAI/HA7PZDLvdHjDX4XDA4/Foc34aPt3j3WM9KSwshM1m08rlct1p20Q0iNxxAOXn5+PkyZPYtm1bf/bTo4KCAni9Xq1qamru+XMS0b1nvJM7rV69Gjt37sSBAweQmJioLXc6nWhra0NjY2PAXlBdXR2cTqc258iRIwGP132WrHvOjSwWCywWy520SkSDWV8OOvv9fsnPz5eEhAT59ttvfzbefRD6448/1padPn1agJ8fhK6rq9PmvPfee2K1WqWlpaVXffAgNIsVHNWvZ8Hy8vLEZrPJvn37pLa2Vqvr169rc1auXClJSUmyZ88eKSkpEbfbLW63WxvvPg0/c+ZMKS8vly+++EJiY2N5Gp7Fug+rXwPoZk+yZcsWbU5zc7OsWrVKhgwZImFhYbJgwQKpra0NeJxz587J7NmzJTQ0VGJiYuS5556T9vZ2BhCLdZ/V7QJI92OwBBWfzwebzaa6DSK6Da/XC6vVetNxXgtGRMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpc0ffB0QPHp0OWLgQMBiAsjLA6wUuXQL8ftWd3Z7DAcybB5w921VXrgBNTaq7IoABRL2k0wHz5wMpKUB7O+DzAefOAd9/3xVIHk/Xm7ulBWhtVd1toNhY4OmnAZGu3i9eBGprgZMnge++66pLl4DmZqCzU3W3/U+n08FsNiMsLAwulwvDhw/HmDFjkJaWhitXruDQoUP45ptvcO7cOVy7dg2dA7gReDU89YpeD3zwAZCa+vMxka7guXIFqKwEXn+9K5AGi7Q04P33u9bhp0S6yucDGhqAnTuB//7v4NiruxmdToeQkBDExcUhMTERqampSEtLQ0pKCkaMGIGoqChERETAYDBAp9MBADo7O9HU1ITvv/8eZWVlOHz4MEpKSnD27Fk0NDTAfxcb5HZXw3MPiPqse0/C5wPq64FTp4DqauCbb7reyDf8HcpBoztwrl/v+gh25kxX38ePd+3J1dYGV/iYzWZERETA6XQiOTkZo0eP1sImPj4e0dHRMBqN0Ol0Wtj0xGAwwG63w263Y8yYMViyZAlaW1tx4cIFVFRUoLi4GEePHkVlZSUaGhrQ2o+7uNwDol7R64H1653w+TwoKekKmW+/7drzaW5W3d2tjRsXiWefFZSVXUVVFXD+fFfYtLR0BelgZzAYYDKZEBsbC5fLhVGjRmHcuHFISUnB8OHDERcXh9DQUJhMplsGzZ0SEbS0tODy5cuorKxESUkJDh06hFOnTuHixYtoa2u76V7S7faAGEDUKwaDAb/73e+wdetW1a30WXp6Ojo6OvDNN9+obqVP4uPjkZ2dDbfbjdGjRyMhIQFRUVEBf6DhXgTOrXTHhd/vx6VLl/Ddd9/h6NGjKCkpwbFjx1BTU4OmpiZtHj+CEQURnU4Hl8uFJUuWYNmyZXjkkUdgMBhUt6XpDjyDwQCHwwGHw4GMjAx0dHTg+vXr+O6773D8+HHs3bu3Vz+sGEBEg4BOp8OwYcOQm5uLp556Cg8//DD0Nx41H6R0Oh1MJhNsNhsmTJiACRMmYMGCBQygwUSn08Fut2PSpEk4c+YMzp8/jyD89Ev9zGAwYOTIkVi+fDlycnLw0EMPBU3w9AcG0D1mNpuRkpKChQsXIjs7G8nJyfB4PHjvvfewdevWm/45arq/GQwGjB49Gk8//TQWL14Mp9P5QAVPNwbQPaDX6+F0OvH4448jJycHbrcbNptN+/w8dOhQvPzyy8jNzcVbb72Fbdu2oaGhQXHXNBCMRiPGjx+PlStXIjs7G9HR0QN+IHkwYQD1o4iICEycOBE5OTmYPXs2EhMTYTKZfjav+/cyRo8ejTfeeAPLli3Dhg0b8Omnn+Lq1asKOqd7zWw2Y+LEicjLy8O8efNgt9sf6ODpxgC6SwaDAcOHD8e8efOwePFijBs3DqGhob1+cRmNRqSnp2PLli04dOgQXn/9dfzrX/9C82D/5RrqFYvFArfbjby8PMyaNQuRkZEMnp9gAN0BnU6H6OhoTJ8+HU888QR+9atfISYm5o4/w3efRXjssccwZcoU7NmzB+vXr8fBgwfR1tbWz93TQAgLC8P06dORl5eHGTNmIDw8nMHTAwZQH1gsFowdOxaLFi3Cb3/7W4wYMaJff/tUp9MhNDQUc+bMwWOPPYbPPvsMr732GsrKygb0AkG6c+Hh4fj1r3+NVatWYfr06QgPD1fd0qDGALoNg8GAhx56CDNnzsQTTzyByZMn3/PdaJ1Oh8jISOTk5GDWrFnYvn07Nm7ciMrKSgbRIGW1WpGVlYVVq1YhIyMDFouFezy9ccu/HH+Dt99+W8aOHSuRkZESGRkpGRkZsmvXLm28ublZVq1aJVFRURIeHi4LFy4Uj8cT8Bjnz5+XOXPmSGhoqMTGxsrzzz8v7e3tfWlDvF6vALinZbVaJTMzUzZv3izV1dXS3t4ufr+/T332F7/fL7W1tfLKK6/II488Ijqd7p6v/41lMBgkNzd3wJ+3Pyo9PV3Gjx/fb4+n1+vFZDJJdHS0pKWlydKlS+Wrr76SlpYWZa+Rwab7Per1em85r097QImJiVi3bh1GjhwJEcHWrVsxf/58lJWVYcyYMfjjH/+Izz77DNu3b4fNZsPq1auxcOFCfPXVVwC6LvufO3cunE4nDh06hNraWixduhQmkwkvv/xyX1rpM4PBALPZDJPJpP3XYrEgMjISVqsVNptNq4SEBMyZMwdjxowZFD/JdDodnE4nnn/+eeTk5GDz5s34xz/+gdraWqV9PQgsFgusVitiYmIwYsQIjBw5EikpKRg1ahQSExMRFRUFq9V62yvOqWd3fTFqVFQUXn31VSxevBixsbH48MMPsXjxYgDA6dOnMXr0aBQXFyMjIwOff/45fvOb3+DixYtwOBwAgHfffRd/+tOf8MMPP8BsNvfqObsvRo2Pj0dkZCTCw8MRERGB8PBwWK1WDBky5Gd147yIiAiEhYXBaDRCr9dr1f09KYP5xdTZ2YmzZ89i06ZN+Oijj/DDDz/c8+e8ny9G1ev1MJlMCA8PR2JiIlwuF1JSUpCamorhw4cjMTERcXFxsFgs2uuFbq37PXrPLkbt7OzE9u3bce3aNbjdbpSWlqK9vR2ZmZnanJSUFCQlJWkBVFxcjLFjx2rhAwBZWVnIy8tDRUUFJkyY0Kce9u7di+joaG2vxmw23/TFMZgDpa8MBgOSk5Px2muv4fe//z02bNiATz75BD6fT3Vrg57JZMKQIUMQHR2N4cOHIyUlBcnJyUhOTsbDDz8Mu92OyMhI7XV0P71uBqM+B9CJEyfgdrvR0tKCiIgI7NixA6mpqSgvL4fZbIbdbg+Y73A4tMsNPB5PQPh0j3eP3Uxra2vAlyB1v9Hi4+Nvma73O6PRiF/84hfYvHkzVqxYgfXr1+Pf//63tidnNBphMBi0uvF2b+Z03zYajYiLi8PSpUtVr3afRUZGIiYmBn/4wx+QnJyM+Ph4OBwOWCyWoNjjvZ/1OYBGjRqF8vJyeL1efPzxx8jNzcX+/fvvRW+awsJCvPTSS/f0OYJV9/f9Tps2Denp6fB6vdryOym9Xn/Tse7HJeovfQ4gs9mMESNGAOj6bH306FG88cYbyMnJQVtbGxobGwP2gurq6uB0OgEATqcTR44cCXi8uh+/v7N7Tk8KCgqwdu1a7bbP54PL5epr6/e9kJAQhISEqG6DqNfu+mia3+9Ha2sr0tPTYTKZUFRUpI1VVVWhuroabrcbAOB2u3HixAnU19drc3bv3g2r1YrUnr7t/EfdZyJ+WkQU/Pq0B1RQUIDZs2cjKSkJTU1N+PDDD7Fv3z58+eWXsNlsWL58OdauXaudmnzmmWfgdruRkZEBAJg5cyZSU1Px1FNP4ZVXXoHH48Ff/vIX5OfnB3zNJBE9GPoUQPX19Vi6dClqa2ths9kwbtw4fPnll3j88ccBAK+//jr0ej0WLVqE1tZWZGVl4e2339bubzAYsHPnTuTl5cHtdiM8PBy5ubn4+9//3r9rRURBIai/lP52v2NARGr09j3K36giImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZo+oG7oSIAAB8Pp/iToioJ93vze736s0EZQBdvnwZAOByuRR3QkS30tTUBJvNdtPxoAygqKgoAEB1dfUtV44C+Xw+uFwu1NTUwGq1qm4nKHCb3RkRQVNTExISEm45LygDSK/vOnRls9n4orgDVquV262PuM36rjc7BzwITUTKMICISJmgDCCLxYIXX3wRFotFdStBhdut77jN7i2d3O48GRHRPRKUe0BEdH9gABGRMgwgIlKGAUREygRlAG3atAlDhw5FSEgIpk6diiNHjqhuSZnCwkJMnjwZkZGRiIuLQ3Z2NqqqqgLmtLS0ID8/H9HR0YiIiMCiRYtQV1cXMKe6uhpz585FWFgY4uLi8MILL6Cjo2MgV0WZdevWQafTYc2aNdoybrMBIkFm27ZtYjab5f3335eKigp5+umnxW63S11dnerWlMjKypItW7bIyZMnpby8XObMmSNJSUly9epVbc7KlSvF5XJJUVGRlJSUSEZGhjz66KPaeEdHh6SlpUlmZqaUlZXJrl27JCYmRgoKClSs0oA6cuSIDB06VMaNGyfPPvustpzbbGAEXQBNmTJF8vPztdudnZ2SkJAghYWFCrsaPOrr6wWA7N+/X0REGhsbxWQyyfbt27U5lZWVAkCKi4tFRGTXrl2i1+vF4/Foc9555x2xWq3S2to6sCswgJqammTkyJGye/du+eUvf6kFELfZwAmqj2BtbW0oLS1FZmamtkyv1yMzMxPFxcUKOxs8vF4vgP9/wW5paSna29sDtllKSgqSkpK0bVZcXIyxY8fC4XBoc7KysuDz+VBRUTGA3Q+s/Px8zJ07N2DbANxmAymoLka9dOkSOjs7A/6nA4DD4cDp06cVdTV4+P1+rFmzBtOmTUNaWhoAwOPxwGw2w263B8x1OBzweDzanJ62affY/Wjbtm04duwYjh49+rMxbrOBE1QBRLeWn5+PkydP4uDBg6pbGdRqamrw7LPPYvfu3QgJCVHdzgMtqD6CxcTEwGAw/OxsRF1dHZxOp6KuBofVq1dj586d2Lt3LxITE7XlTqcTbW1taGxsDJj/023mdDp73KbdY/eb0tJS1NfXY+LEiTAajTAajdi/fz82btwIo9EIh8PBbTZAgiqAzGYz0tPTUVRUpC3z+/0oKiqC2+1W2Jk6IoLVq1djx44d2LNnD4YNGxYwnp6eDpPJFLDNqqqqUF1drW0zt9uNEydOoL6+Xpuze/duWK1WpKamDsyKDKAZM2bgxIkTKC8v12rSpElYsmSJ9m9uswGi+ih4X23btk0sFot88MEHcurUKVmxYoXY7faAsxEPkry8PLHZbLJv3z6pra3V6vr169qclStXSlJSkuzZs0dKSkrE7XaL2+3WxrtPKc+cOVPKy8vliy++kNjY2AfqlPJPz4KJcJsNlKALIBGRN998U5KSksRsNsuUKVPk66+/Vt2SMgB6rC1btmhzmpubZdWqVTJkyBAJCwuTBQsWSG1tbcDjnDt3TmbPni2hoaESExMjzz33nLS3tw/w2qhzYwBxmw0Mfh0HESkTVMeAiOj+wgAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiU+X+eHPeq5CLVpgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.5       -1.5       -5.        -5.        -3.1415927 -5.\n",
      " -0.        -0.       ], [1.5       1.5       5.        5.        3.1415927 5.        1.\n",
      " 1.       ], (8,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (2,), float32)\n",
      "state= [-0.00274715  1.4168403  -0.27828535  0.2631164   0.00319017  0.06303567\n",
      "  0.          0.        ]\n",
      "action= [-0.97116625 -0.24147968]\n",
      "next_state= [-0.00549479  1.4221826  -0.2779088   0.23741789  0.00630512  0.06230493\n",
      "  0.          0.        ]\n",
      "reward= 0.8999845656363163\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7ff4f3cd9760>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-281.24740603069074, 167.4010756187529)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/2.PPO.Lunar Lander Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(272.2415943570289, 54.26272932228687)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/2.PPO.Lunar Lander Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "414e1478",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAacUlEQVR4nO3de1BU990/8PfZKyDsrrCwK2ERoigq3ooIq5lJ+4iisRZvvVhbedL8klHRxuo4LTOtmfSPkkmeJjbGmPSPGn/tz2A0Y02swTF4e5JQjAQi3tDmqYFEdvHGLgosl/3+/iCcxzWoILBfFt+vme8A53w553O+cN6c2y6KEEKAiEgCjewCiOjhxQAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJppAXQ1q1bkZSUhLCwMGRmZuLEiROySiEiSaQE0K5du7B+/Xo899xz+OyzzzB58mTk5OSgvr5eRjlEJIki48WomZmZyMjIwGuvvQYA8Pv9cDgcWLt2LX7zm98EuxwikkQX7BW2traivLwcBQUF6jSNRoPs7GyUlpZ2+z0+nw8+n0/92u/34/r164iJiYGiKANeMxH1jhACjY2NiI+Ph0Zz9xOtoAfQ1atX0dHRAZvNFjDdZrPh/Pnz3X5PYWEhnn/++WCUR0T9qLa2FgkJCXedHxJ3wQoKCuDxeNRWU1MjuyQi6oGoqKh7zg/6EZDVaoVWq4Xb7Q6Y7na7Ybfbu/0eo9EIo9EYjPKIqB/d7xJJ0I+ADAYD0tPTUVJSok7z+/0oKSmB0+kMdjlEJFHQj4AAYP369cjLy8O0adMwffp0bN68Gbdu3cKTTz4poxwikkRKAP34xz/GlStXsGnTJrhcLkyZMgXFxcXfujBNREOblOeA+srr9cJsNssug4juw+PxwGQy3XV+SNwFI6KhiQFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnT6wA6fvw4FixYgPj4eCiKgr///e8B84UQ2LRpE0aMGIHw8HBkZ2fj4sWLAX2uX7+O5cuXw2QywWKx4KmnnsLNmzf7tCFEFHp6HUC3bt3C5MmTsXXr1m7nv/jii3j11VfxxhtvoKysDMOGDUNOTg5aWlrUPsuXL8eZM2dw6NAh7N+/H8ePH8czzzzz4FtBRKFJ9AEAsXfvXvVrv98v7Ha7eOmll9RpDQ0Nwmg0irffflsIIcTZs2cFAPHpp5+qfT744AOhKIr4+uuve7Rej8cjALCxsQ3y5vF47rkv9+s1oH//+99wuVzIzs5Wp5nNZmRmZqK0tBQAUFpaCovFgmnTpql9srOzodFoUFZW1u1yfT4fvF5vQCOi0NevAeRyuQAANpstYLrNZlPnuVwuxMXFBczX6XSIjo5W+9ypsLAQZrNZbQ6Hoz/LJiJJQuIuWEFBATwej9pqa2tll0RE/aBfA8hutwMA3G53wHS3263Os9vtqK+vD5jf3t6O69evq33uZDQaYTKZAhoRhb5+DaDk5GTY7XaUlJSo07xeL8rKyuB0OgEATqcTDQ0NKC8vV/scPnwYfr8fmZmZ/VkOEQ12vbjpJYQQorGxUVRUVIiKigoBQLz88suioqJCfPnll0IIIV544QVhsVjEvn37xKlTp0Rubq5ITk4Wzc3N6jLmzp0rpk6dKsrKysRHH30kUlJSxLJly3pcA++CsbGFRrvfXbBeB9CRI0e6XVFeXp4QovNW/O9+9zths9mE0WgUs2bNEtXV1QHLuHbtmli2bJmIjIwUJpNJPPnkk6KxsZEBxMY2xNr9AkgRQgiEGK/XC7PZLLsMIroPj8dzz2u2IXEXjIiGJgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCRNrwKosLAQGRkZiIqKQlxcHBYuXIjq6uqAPi0tLcjPz0dMTAwiIyOxZMkSuN3ugD41NTWYP38+IiIiEBcXh40bN6K9vb3vW0NEoUX0Qk5Ojti+fbs4ffq0qKysFE888YRITEwUN2/eVPusXLlSOBwOUVJSIk6ePCmysrLEjBkz1Pnt7e0iLS1NZGdni4qKCnHgwAFhtVpFQUFBj+vweDwCABsb2yBvHo/nnvtyrwLoTvX19QKAOHbsmBBCiIaGBqHX68Xu3bvVPufOnRMARGlpqRBCiAMHDgiNRiNcLpfaZ9u2bcJkMgmfz9ej9TKA2NhCo90vgPp0Dcjj8QAAoqOjAQDl5eVoa2tDdna22ic1NRWJiYkoLS0FAJSWlmLixImw2Wxqn5ycHHi9Xpw5c6bb9fh8Pni93oBGRKHvgQPI7/dj3bp1mDlzJtLS0gAALpcLBoMBFosloK/NZoPL5VL73B4+XfO75nWnsLAQZrNZbQ6H40HLJqJB5IEDKD8/H6dPn0ZRUVF/1tOtgoICeDwetdXW1g74Oolo4Oke5JvWrFmD/fv34/jx40hISFCn2+12tLa2oqGhIeAoyO12w263q31OnDgRsLyuu2Rdfe5kNBphNBofpFQiGsx6c9HZ7/eL/Px8ER8fLy5cuPCt+V0Xoffs2aNOO3/+vAC+fRHa7Xarfd58801hMplES0tLj+rgRWg2ttBo/XoXbNWqVcJsNoujR4+Kuro6tTU1Nal9Vq5cKRITE8Xhw4fFyZMnhdPpFE6nU53fdRt+zpw5orKyUhQXF4vY2FjehmdjG4KtXwPobivZvn272qe5uVmsXr1aDB8+XERERIhFixaJurq6gOVcunRJzJs3T4SHhwur1So2bNgg2traGEBsbEOs3S+AlG+CJaR4vV6YzWbZZRDRfXg8HphMprvO52vBiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaR7o/YDo4aMowOLFgFYLVFQAHg9w9Srg98uu7P5sNmDBAuCLLzrbjRtAY6PsqghgAFEPKQqQmwukpgJtbYDXC1y6BHz1VWcguVydO3dLC+Dzya42UGws8PTTgBCdtV++DNTVAadPA//zP53t6lWguRno6JBd7cOFAUS9otEARmPnTh0bC2RkAAsXdgbPjRvAuXPAK690BtJgoiidR286HTBqVGebObMzlLxe4Pp1YP9+4G9/C42juqGCAUS91nUk4fUC9fXA2bNATQ3w+eedO/Id/4dy0BCiszU1dZ6CXbzYWfepU51HcnV1DJ9gYwBRj12/bseBAy6cPNkZMhcudB75NDfLruzeNJooVFUJVFTcRHU18OWXnWHT0tIZpCQPA4h6RFG0qKjIwY4dO2SX0mt6/Ri8+mo7Pv/8c9ml0B0YQDQgkkdmITlhZucbc0J880F0fhbwEfDc/ApV596H398utWYKvpAOoKSkJNTU1MAfIifuWq0W8fHxuHbtGpqammSXM2D0unBkTvw/GDMiBwqUu/b7JpZQ7z2DL/7937jZdDVYJdIgEdIPIh48eBB//etfsWDBAvXfQw82Go0GNpsNP/rRj/D222+jrKwM+/btw9y5cxEWFia7vAGh0egQpo+CAg0U5e5No2ihUbQw6k3Q6YfmWNC9hfQRkN1ux7Jly7B48WJcuHAB+/btw7vvvotz586htbVVam1RUVFIT0/HkiVLMHfuXCQmJsJgMKh1O51OFBcX45VXXkFZWRna24fO6Ud87ESYIxPu3/Eb4XoLEuMz0OD5agCromDSaDQ9OjMJ6QACAEVREBYWhkmTJmHixIlYu3YtysrKUFRUhEOHDuHy5csI1j/+CAsLw9ixY5Gbm4sf/OAHmDBhAoxGIxQl8DREURQMGzYMixcvxuzZs7F3715s3rwZp06dCpnTyXvxow0GXdS3tvtudNpwKJqe9aXBKTw8HLGxsZgyZQoyMjIwadIk5Obm3vf7Qj6AbqcoCiwWC+bMmYNZs2ahtrYWxcXFeOedd3Dy5EncvHmz39ep1WrhcDgwe/ZsLF26FBkZGTCbzdBo7n92qygKTCYTVqxYgfnz52Pnzp14/fXXcfHixZAOogR7OvSa8B731yg6JNqno+rce7wQHQI0Gg3Cw8ORlJSEtLQ0zJgxA1OmTEFqairMZjMMBgMae/halyEVQF0URYFOp0NycjJWrlyJvLw8VFVVYc+ePXj//ffxr3/9Cx19fObearXiscceww9/+EN897vfhd1uh6IoPf6rf2e9MTExWLt2LZYsWYK33noLf/7zn1FTU9OnGmWJioyDVtH3uL8CDaIi4h5o7GjgKYqC6OhoPPLII8jKysL06dORkZEBh8MBs9ms/twe5Oc3JAPodoqiICIiQh20jRs34tixYygqKsLx48dx7dq1Hp+iRUZGYurUqVi0aBHmzZuHRx99FHq9vl92nK5lPPLIIygoKMBPfvITvPnmm9ixYwfq6+v7vPyBFhExHADQ3OyBX7RDUXr3q6XV6KHV6qHVGhAebobHU4dv7uFTkBkMBpjNZowbNw7p6emYMWMG0tLS4HA4EBYWBo1G029/LIZ8AHXpOjqJi4vD0qVLkZubiy+++ALvvfce3n33XZw6dQq+bl5FqdfrkZKSggULFmDhwoWYOHEiIiIiBvSvtUajwahRo1BYWIi8vDxs2bIFu3btQkNDw4Ctsy8URUHm5P9EfOwUHPn0vzrvfvXy+wEF1ujReCx9Jfz+Drz34a/R2jZ0H1UYTAwGA0aMGIHU1FRkZWUhIyMDaWlpiI2NRXh4+ID+rj80AXQ7RVFgMBiQmpqK1NRUrF69GidOnMA777yD4uJiuFwu2Gw2ZGdnY+nSpcjKyoLFYnngU6wHpdVqMX78eLz22mv4xS9+gc2bN2P//v09Pr8OFovJgXFJ89He7oPH8/UDLUNAwOe7ichwO4ZHjERMdDLq3Gf6udKHm9FoRHh4OMxmMxwOB0aOHIkpU6YgMzMTo0ePhtVqhU7XGQnB+j3vVQBt27YN27Ztw6VLlwAAEyZMwKZNmzBv3jwAQEtLCzZs2ICioiL4fD7k5OTg9ddfh81mU5dRU1ODVatW4ciRI4iMjEReXh4KCwvVDQ+mrkGOiorCrFmz8Pjjj6Ourg5VVVVIS0tDfHy8lLrurFGn02H69OnYvn07Pv74Y7z88sv48MMPuz1ik8ERn46osBEov/B/0dTcgHa/D7fartz2EGJ3v8wKbv8db+tows1b9bhcfwq20RPwqOOxQRtAGo0GWq0WJpMJI0aMgMlkQmNjI7xeLzweD5qamuD3+9UWLIqiQKPRwGAwIDo6GjabDQkJCUhJSUFKSgqSkpLgcDhgs9kQFhaGsLAwaLXaoNXXnV7tXQkJCXjhhReQkpICIQR27NiB3NxcVFRUYMKECfjVr36Ff/zjH9i9ezfMZjPWrFmDxYsX4+OPPwYAdHR0YP78+bDb7fjkk09QV1eHFStWQK/X4w9/+MOAbGBv6HQ6OBwOOBwO2aV0y2g04nvf+x6cTic+/PBD/PGPf8Qnn3yCNomvqDToIzExJRdCCFysOQIh/Lh8uQrNt95E99dwRLefNbVcR1t7C2ouf4pxSU8gxfEfOFn1/+Dz9f+dy94wGo0wmUywWq0YPXo0UlJSkJqairFjxyIhIQHR0dEYNmwYmpqa0NTUhFu3bqGhoQFutxsulwsulwtutxv19fXqR4/Hg5aWFrS0tKC5ufmBQio8PByRkZEYPnw4kpKS8Oijj2LUqFEYPXo0kpOTERMTg+HDhwecQg3Gi/yK6ONDMtHR0XjppZewdOlSxMbGYufOnVi6dCkA4Pz58xg3bhxKS0uRlZWFDz74AN///vdx+fJl9ajojTfewK9//WtcuXJFfVDvfrxeL8xmMzweD0wmU1/KD1lCCDQ2NuL999/H5s2bUVlZOaAPM2q1WvzsZz/71otRRz6Sidzv/RGNTS68feAXaGnx9mk9ligHFs/ZDGtUCoo+fBI1X5X3aXkAkJ6ejvb2u78YVaPRQK/XY9iwYUhISIDD4UBqairGjx+PUaNGISEhAXFxcTAajdDpdD16xKKLEAJCCHR0dKjt1q1buHHjBm7cuIHr16/jypUruHz5Murq6lBXV4erV6+q8xsbGxEdHY34+HgkJiZizJgxSElJgcPhQHx8PGJjY2EwGNS6BkvI9HQffeDzi46ODuzevRu3bt2C0+lEeXk52trakJ2drfZJTU1FYmKiGkClpaWYOHFiwClZTk4OVq1ahTNnzmDq1KkPWs5Dp+sZop/+9KeYO3cudu3ahS1btqC6ujpoD14CwLhRc6HVGPCvrw7D5+v7tSnPzctwXT0LS2QiRj4yHbVfV0CI/juN0ev1GD58OGJiYjBq1CikpqZizJgxGDNmDEaOHAmLxYKoqCg1ZPq6Q3ddN+wKOQAYNmwY4uLi1D53/rxaW1vR3NyMlpYWtLa2IiwsDFFRUQEv3RksQdNXvQ6gqqoqOJ1OtLS0IDIyEnv37sX48eNRWVkJg8EAi8US0N9ms8H1zdvjdV3cvXN+17y78fl8Adc7vN6+/ZUdSrqeIVq1ahUWL16M8+fPo62tDW1tbWhvbw9ofZnW3t4Oq9WKFStWBKw/PjYK8fYb0FrjkZz6837ZppHx0bBYvkKHKRrJY3/W5+VFRUXBarXil7/8JcaMGYMRI0bAZrPBaDRCq9UG/ebCne5ct9FohNFolFRNcPU6gMaOHYvKykp4PB7s2bMHeXl5OHbs2EDUpiosLMTzzz8/oOsIdYqiwG63w263S6pgzgAsc+EALJMGk16/Gt5gMGD06NFIT09HYWEhJk+ejD/96U+w2+1obW391rMqbrdb3Snsdjvcd7xfZ9fX99pxCgoK4PF41FZbW9vbsoloEOrz23H4/X74fD6kp6dDr9ejpKREnVddXY2amho4nU4AgNPpRFVVVcCTvYcOHYLJZML48ePvuo6uOxG3NyIKfb06BSsoKMC8efOQmJiIxsZG7Ny5E0ePHsXBgwdhNpvx1FNPYf369YiOjobJZMLatWvhdDqRlZUFAJgzZw7Gjx+Pn//853jxxRfhcrnw29/+Fvn5+Q/NOS8R/a9eBVB9fT1WrFiBuro6mM1mTJo0CQcPHsTs2bMBAK+88go0Gg2WLFkS8CBiF61Wi/3792PVqlVwOp0YNmwY8vLy8Pvf/75/t4qIQkKfnwOSgc8BEQ1uPd1HQ/otWYkotDGAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQ62QU8CCEEAMDr9UquhIi607Vvdu2rdxOSAXTt2jUAgMPhkFwJEd1LY2MjzGbzXeeHZABFR0cDAGpqau65cRTI6/XC4XCgtrYWJpNJdjkhgWP2YIQQaGxsRHx8/D37hWQAaTSdl67MZjN/KR6AyWTiuPUSx6z3enJwwIvQRCQNA4iIpAnJADIajXjuuedgNBpllxJSOG69xzEbWIq4330yIqIBEpJHQEQ0NDCAiEgaBhARScMAIiJpQjKAtm7diqSkJISFhSEzMxMnTpyQXZI0hYWFyMjIQFRUFOLi4rBw4UJUV1cH9GlpaUF+fj5iYmIQGRmJJUuWwO12B/SpqanB/PnzERERgbi4OGzcuBHt7e3B3BRpXnjhBSiKgnXr1qnTOGZBIkJMUVGRMBgM4i9/+Ys4c+aMePrpp4XFYhFut1t2aVLk5OSI7du3i9OnT4vKykrxxBNPiMTERHHz5k21z8qVK4XD4RAlJSXi5MmTIisrS8yYMUOd397eLtLS0kR2draoqKgQBw4cEFarVRQUFMjYpKA6ceKESEpKEpMmTRLPPvusOp1jFhwhF0DTp08X+fn56tcdHR0iPj5eFBYWSqxq8KivrxcAxLFjx4QQQjQ0NAi9Xi92796t9jl37pwAIEpLS4UQQhw4cEBoNBrhcrnUPtu2bRMmk0n4fL7gbkAQNTY2ipSUFHHo0CHx+OOPqwHEMQuekDoFa21tRXl5ObKzs9VpGo0G2dnZKC0tlVjZ4OHxeAD87wt2y8vL0dbWFjBmqampSExMVMestLQUEydOhM1mU/vk5OTA6/XizJkzQaw+uPLz8zF//vyAsQE4ZsEUUi9GvXr1Kjo6OgJ+6ABgs9lw/vx5SVUNHn6/H+vWrcPMmTORlpYGAHC5XDAYDLBYLAF9bTYbXC6X2qe7Me2aNxQVFRXhs88+w6effvqteRyz4AmpAKJ7y8/Px+nTp/HRRx/JLmVQq62txbPPPotDhw4hLCxMdjkPtZA6BbNardBqtd+6G+F2u2G32yVVNTisWbMG+/fvx5EjR5CQkKBOt9vtaG1tRUNDQ0D/28fMbrd3O6Zd84aa8vJy1NfX4zvf+Q50Oh10Oh2OHTuGV199FTqdDjabjWMWJCEVQAaDAenp6SgpKVGn+f1+lJSUwOl0SqxMHiEE1qxZg7179+Lw4cNITk4OmJ+eng69Xh8wZtXV1aipqVHHzOl0oqqqCvX19WqfQ4cOwWQyYfz48cHZkCCaNWsWqqqqUFlZqbZp06Zh+fLl6uccsyCRfRW8t4qKioTRaBRvvfWWOHv2rHjmmWeExWIJuBvxMFm1apUwm83i6NGjoq6uTm1NTU1qn5UrV4rExERx+PBhcfLkSeF0OoXT6VTnd91SnjNnjqisrBTFxcUiNjb2obqlfPtdMCE4ZsEScgEkhBBbtmwRiYmJwmAwiOnTp4t//vOfskuSBkC3bfv27Wqf5uZmsXr1ajF8+HAREREhFi1aJOrq6gKWc+nSJTFv3jwRHh4urFar2LBhg2hrawvy1shzZwBxzIKDb8dBRNKE1DUgIhpaGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0/x++Hnm6z6v1AQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "288.2618139933845 215 [0.7179384502343567, 0.7686211759246362, 0.7738843312328356, 1.1586963072128913, 0.8596935515823816, 1.3921985670843389, 0.9493888533870631, 1.3976901913345388, 0.9703761366013168, 0.8998041733740934, 1.2683941418709004, 1.510405461603866, 0.842608355102783, 0.708108347815255, 1.0484193793252166, 0.48942223080354097, 0.9273259623663569, 0.2538313558979439, 0.0836153128059891, -0.07845104111160595, -0.3990694225088305, -1.0524983026904806, -1.1703181849802604, -1.2707567442189145, -1.4085500669297915, -1.4240330197047228, -1.4757410865840086, -1.5140897391261774, -1.5405118062132033, -1.5564933197917696, -1.5633357537496977, -1.5622877977783673, -1.5543552638798417, -1.5405068301204778, -1.3411792527498971, -1.664892654173002, -1.6366451607187855, -1.6051878593306128, -1.570909206838138, -1.5342051800569436, -1.4088987336992158, -1.5819873499792152, -1.5392858458108662, -1.4953000244149166, -1.4501340060389794, -1.4040224692284369, 0.6441812189413441, 2.1204467392581647, 1.1293172724021985, 2.3184826518752515, 2.5585768709761054, 2.061331993967104, 3.112408219693492, -1.5785635467129566, 2.0513594140343208, 0.9428336560690809, 0.6783775845506852, 3.1226172618119277, 1.8422163880135998, 2.338941243152135, 1.44663618379289, 1.4260231622176947, 0.37434109131196236, 2.1523451158143985, -1.5698193817607944, 1.348316383635762, -1.473310305207292, 2.08923693599981, 1.5732964694318696, 1.2670747469215597, 2.3470060986341537, 0.49030881190534725, 1.8997467935072336, 2.1315437188384068, 3.194277697778034, 0.9766613487701079, -0.024988838123795182, 0.7642279787567361, 1.9808772645730186, 0.08484358681008644, 2.9460887928190855, 0.7193804201615478, 3.4796184632929625, 1.7344584687351812, 0.8643341264046569, 3.2311252410602593, 2.3552207902182603, 0.8824625616726337, 0.16068077929802485, 1.3325114759246184, 2.7890378350321496, 1.2274909366660154, 3.6223986401481945, 1.302947925108964, 0.9823061440391427, 0.9748000497264343, 1.09881802098364, 1.1344638197223276, 2.198359936489163, 1.7367853554294812, 2.8614302542221743, 0.8340010658154086, 3.1865853406752933, 1.485292892740776, 2.1180202471212595, 1.783545806280506, 2.4958490423832473, 2.2790998728164285, 1.7874859192007306, 2.487180136035559, 2.0894912926050377, 1.6870563440209765, 1.6965029782436176, 0.6200464012341967, 0.8523449325013559, 1.7710987354830565, 3.361887600504235, 2.159621650385887, 0.056843810982002196, 1.9913715205293954, 4.470027631514176, 1.638139591851513, 1.3845677827055283, -0.8745479483669214, 0.010585973774976776, 0.2922415592109502, 0.7986450535387718, -0.31114545361556906, -1.2830720795555113, 1.2847716897016948, -1.4992447773568216, 0.7659208358711538, 1.1309106359719352, -1.1934012380261094, -0.2896456942304548, 2.628856308448275, 2.168072339320389, 1.0971220815893628, -1.1921042181591646, -1.2966512232852807, 0.2596997625190113, 1.5077802441495407, 1.919628714177034, 1.863054843587534, 1.2731934561105347, 1.4551541806142836, 0.958921755361412, 2.7311454141190694, -1.004712399030292, 1.5417906714365233, 2.4926743079176577, 10.406783065701031, 2.2217067629086813, 6.687189558465912, 6.401488243057863, 3.9803839597736856, -5.31786109981937, -0.8420456070899105, 6.474460969601176, 0.18934595422696845, 2.2089838643450506, 2.9711191431063675, 13.264224417758939, -0.5097932258801038, 1.8770254339225587, 0.9641878321596131, 1.189996412687897, 0.8144038785755185, 0.34735229963718695, 0.11415592280400233, 0.15552951548808736, 1.53942134193532, 1.1175509546285483, 0.8702736298975502, 0.29285892720055173, 0.47496307503299917, 0.4944925716969557, 0.493824537891804, 0.4993158926546162, 0.5049838682734968, 0.510652640206203, 0.5163140058957314, 0.5219678312345053, 0.5276237241483148, 0.5332872410327578, 0.53894369338955, 0.5445997090913544, 0.5502536000758198, 0.5559102069767761, 0.5615650572912543, 0.3609269833667703, -0.11780515478095488, 0.07409139623798033, 0.08669808583812255, -0.0203477661207927, 0.007361911810445321, 0.015260096706992243, 0.0001317334435242401, 1.3939452085054427e-05, 0.0024804739737458448, 0.0007674590261590453, -0.00023954457155639375, 0.0003730943423150279, 0.00027077588202040204, -9.013708216798477e-05, 4.47445220306264e-05, 5.804171074430542e-05, -8.480880962835613e-06, 3.6061534185449773e-06, 9.00688866600774e-06, 4.023555588261729e-06, -3.5570188465783303e-06, -1.0231927447534872e-06, 1.0507847623131283e-06, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
